import urllib.request
import random
import re
class JD:
  def __init__(self):
      self.data=""
      self.i=""
      self.uapools=[
        'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3',
        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3',
        'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
          ]
      self.ua=random.choice(self.uapools)
      self.ip="127.0.0.1:8888"
      self.thisip=urllib.request.ProxyHandler({"http":self.ip})
      self.opener=urllib.request.build_opener(self.thisip,urllib.request.HTTPHandler)
      self.headers=("User-Agent",self.ua)
      self.opener.addheaders=[self.headers]
      urllib.request.install_opener(self.opener)
  def snacks(self):
       for self.i in range(1,36):
          print("正在爬第"+str(self.i)+"页零食以及评论")
          self.url="https://search.jd.com/search?keyword=%E9%9B%B6%E9%A3%9F&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&cid2=1583&cid3=1590&page="+str(self.i*2-1)
          self.data=urllib.request.urlopen(self.url).read().decode("utf-8","ignore")
          self.comments()
          self.link='class="p-name p-name-type-2.*? href="(.*?)"'
          self.price='class="p-price".*?<i>(.*?)</i>'
          self.shop='class="p-shop".*?title="(.*?)">'
          self.cnums='class="p-commit".*?<a .*?">(.*?)</a>'
          thislink=re.compile(self.link,re.S).findall(self.data)
          thisprice=re.compile(self.price,re.S).findall(self.data)
          thisshop=re.compile(self.shop,re.S).findall(self.data)
          thiscnums=re.compile(self.cnums,re.S).findall(self.data)
          for k in range(0,len(thislink)):
              try:
                thisurl=thislink[k]
                if(thisurl.find("https:")==-1):
                  thisurl="https:"+thisurl
                thisdata=urllib.request.urlopen(thisurl).read().decode("gbk","ignore")
                name='<title>(.*?)</title>'
                self.thisname=re.compile(name).findall(thisdata)
                fh=open("F:\\第三周作业\\京东零食.txt","a")
                if(len(thisshop[k])<=20):
                  pass
                else:
                  thisshop[k]="京东自营"
                a={"商品名称":str(self.thisname[0]),"商品价格":str(thisprice[k]),"商品出售方":str(thisshop[k]),"商品评论数":str(thiscnums[k])}
                fh.write(str(a)+'\n\n')
                fh.close()
              except Exception as err:
                print(err)
  def comments(self):
        pat='<div class="p-commit">.*?href="//item.jd.com/(.*?).html'
        rst=re.compile(pat,re.S).findall(self.data)
        for j in range(0,len(rst)):
            try:
                fh1=open("F:\\第三周作业\\零食评论\\第"+str(self.i)+"页第"+str(j+1)+"个零食评论"+".txt","w")
                pid=rst[j]
                for i in range(1,3):
                    curl="https://sclub.jd.com/comment/productPageComments.action?&productId="+str(pid)+"&score=0&sortType=5&page="+str(i)+"&pageSize=10"
                    cdata=urllib.request.urlopen(curl).read().decode("gbk","ignore")
                    comments='{"id":.*?,"topped":.*?,"content":"(.*?)"'
                    crst=re.compile(comments).findall(cdata)
                    for k in range(0,len(crst)):
                        thiscomment=crst[k]
                        fh1.write("第"+str(i)+"页第"+str(k+1)+"条评论： "+thiscomment+'\n\n')
                fh1.close()
            except Exception as err:
                print(err)
jd=JD()
jd.snacks()
jd.comments()
